{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "import random\n",
    "\n",
    "import gym\n",
    "import numpy as np\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F\n",
    "from torch.distributions import Normal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.display import clear_output\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Use CUDA</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "use_cuda = torch.cuda.is_available()\n",
    "device   = torch.device(\"cuda\" if use_cuda else \"cpu\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Replay Buffer</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ReplayBuffer:\n",
    "    def __init__(self, capacity):\n",
    "        self.capacity = capacity\n",
    "        self.buffer = []\n",
    "        self.position = 0\n",
    "    \n",
    "    def push(self, state, action, reward, next_state, done):\n",
    "        if len(self.buffer) < self.capacity:\n",
    "            self.buffer.append(None)\n",
    "        self.buffer[self.position] = (state, action, reward, next_state, done)\n",
    "        self.position = (self.position + 1) % self.capacity\n",
    "    \n",
    "    def sample(self, batch_size):\n",
    "        batch = random.sample(self.buffer, batch_size)\n",
    "        state, action, reward, next_state, done = map(np.stack, zip(*batch))\n",
    "        return state, action, reward, next_state, done\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.buffer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class NormalizedActions(gym.ActionWrapper):\n",
    "    def _action(self, action):\n",
    "        low  = self.action_space.low\n",
    "        high = self.action_space.high\n",
    "        \n",
    "        action = low + (action + 1.0) * 0.5 * (high - low)\n",
    "        action = np.clip(action, low, high)\n",
    "        \n",
    "        return action\n",
    "\n",
    "    def _reverse_action(self, action):\n",
    "        low  = self.action_space.low\n",
    "        high = self.action_space.high\n",
    "        \n",
    "        action = 2 * (action - low) / (high - low) - 1\n",
    "        action = np.clip(action, low, high)\n",
    "        \n",
    "        return actions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot(frame_idx, rewards):\n",
    "    clear_output(True)\n",
    "    plt.figure(figsize=(20,5))\n",
    "    plt.subplot(131)\n",
    "    plt.title('frame %s. reward: %s' % (frame_idx, rewards[-1]))\n",
    "    plt.plot(rewards)\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1>Soft Actor-Critic: Off-Policy Maximum Entropy Deep Reinforcement Learning with a Stochastic Actor</h1>\n",
    "<h2><a href=\"https://arxiv.org/abs/1801.01290\">Arxiv</a></h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ValueNetwork(nn.Module):\n",
    "    def __init__(self, state_dim, hidden_dim, init_w=3e-3):\n",
    "        super(ValueNetwork, self).__init__()\n",
    "        \n",
    "        self.linear1 = nn.Linear(state_dim, hidden_dim)\n",
    "        self.linear2 = nn.Linear(hidden_dim, hidden_dim)\n",
    "        self.linear3 = nn.Linear(hidden_dim, 1)\n",
    "        \n",
    "        self.linear3.weight.data.uniform_(-init_w, init_w)\n",
    "        self.linear3.bias.data.uniform_(-init_w, init_w)\n",
    "        \n",
    "    def forward(self, state):\n",
    "        x = F.relu(self.linear1(state))\n",
    "        x = F.relu(self.linear2(x))\n",
    "        x = self.linear3(x)\n",
    "        return x\n",
    "        \n",
    "        \n",
    "class SoftQNetwork(nn.Module):\n",
    "    def __init__(self, num_inputs, num_actions, hidden_size, init_w=3e-3):\n",
    "        super(SoftQNetwork, self).__init__()\n",
    "        \n",
    "        self.linear1 = nn.Linear(num_inputs + num_actions, hidden_size)\n",
    "        self.linear2 = nn.Linear(hidden_size, hidden_size)\n",
    "        self.linear3 = nn.Linear(hidden_size, 1)\n",
    "        \n",
    "        self.linear3.weight.data.uniform_(-init_w, init_w)\n",
    "        self.linear3.bias.data.uniform_(-init_w, init_w)\n",
    "        \n",
    "    def forward(self, state, action):\n",
    "        x = torch.cat([state, action], 1)\n",
    "        x = F.relu(self.linear1(x))\n",
    "        x = F.relu(self.linear2(x))\n",
    "        x = self.linear3(x)\n",
    "        return x\n",
    "        \n",
    "        \n",
    "class PolicyNetwork(nn.Module):\n",
    "    def __init__(self, num_inputs, num_actions, hidden_size, init_w=3e-3, log_std_min=-20, log_std_max=2):\n",
    "        super(PolicyNetwork, self).__init__()\n",
    "        \n",
    "        self.log_std_min = log_std_min\n",
    "        self.log_std_max = log_std_max\n",
    "        \n",
    "        self.linear1 = nn.Linear(num_inputs, hidden_size)\n",
    "        self.linear2 = nn.Linear(hidden_size, hidden_size)\n",
    "        \n",
    "        self.mean_linear = nn.Linear(hidden_size, num_actions)\n",
    "        self.mean_linear.weight.data.uniform_(-init_w, init_w)\n",
    "        self.mean_linear.bias.data.uniform_(-init_w, init_w)\n",
    "        \n",
    "        self.log_std_linear = nn.Linear(hidden_size, num_actions)\n",
    "        self.log_std_linear.weight.data.uniform_(-init_w, init_w)\n",
    "        self.log_std_linear.bias.data.uniform_(-init_w, init_w)\n",
    "        \n",
    "    def forward(self, state):\n",
    "        x = F.relu(self.linear1(state))\n",
    "        x = F.relu(self.linear2(x))\n",
    "        \n",
    "        mean    = self.mean_linear(x)\n",
    "        log_std = self.log_std_linear(x)\n",
    "        log_std = torch.clamp(log_std, self.log_std_min, self.log_std_max)\n",
    "        \n",
    "        return mean, log_std\n",
    "    \n",
    "    def evaluate(self, state, epsilon=1e-6):\n",
    "        mean, log_std = self.forward(state)\n",
    "        std = log_std.exp()\n",
    "        \n",
    "        normal = Normal(mean, std)\n",
    "        z = normal.sample()\n",
    "        action = torch.tanh(z)\n",
    "        \n",
    "        log_prob = normal.log_prob(z) - torch.log(1 - action.pow(2) + epsilon)\n",
    "        log_prob = log_prob.sum(-1, keepdim=True)\n",
    "        \n",
    "        return action, log_prob, z, mean, log_std\n",
    "        \n",
    "    \n",
    "    def get_action(self, state):\n",
    "        state = torch.FloatTensor(state).unsqueeze(0).to(device)\n",
    "        mean, log_std = self.forward(state)\n",
    "        std = log_std.exp()\n",
    "        \n",
    "        normal = Normal(mean, std)\n",
    "        z      = normal.sample()\n",
    "        action = torch.tanh(z)\n",
    "        \n",
    "        action  = action.detach().cpu().numpy()\n",
    "        return action[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def soft_q_update(batch_size, \n",
    "           gamma=0.99,\n",
    "           mean_lambda=1e-3,\n",
    "           std_lambda=1e-3,\n",
    "           z_lambda=0.0,\n",
    "           soft_tau=1e-2,\n",
    "          ):\n",
    "    state, action, reward, next_state, done = replay_buffer.sample(batch_size)\n",
    "\n",
    "    state      = torch.FloatTensor(state).to(device)\n",
    "    next_state = torch.FloatTensor(next_state).to(device)\n",
    "    action     = torch.FloatTensor(action).to(device)\n",
    "    reward     = torch.FloatTensor(reward).unsqueeze(1).to(device)\n",
    "    done       = torch.FloatTensor(np.float32(done)).unsqueeze(1).to(device)\n",
    "\n",
    "    expected_q_value = soft_q_net(state, action)\n",
    "    expected_value   = value_net(state)\n",
    "    new_action, log_prob, z, mean, log_std = policy_net.evaluate(state)\n",
    "\n",
    "\n",
    "    target_value = target_value_net(next_state)\n",
    "    next_q_value = reward + (1 - done) * gamma * target_value\n",
    "    q_value_loss = soft_q_criterion(expected_q_value, next_q_value.detach())\n",
    "\n",
    "    expected_new_q_value = soft_q_net(state, new_action)\n",
    "    next_value = expected_new_q_value - log_prob\n",
    "    value_loss = value_criterion(expected_value, next_value.detach())\n",
    "\n",
    "    log_prob_target = expected_new_q_value - expected_value\n",
    "    policy_loss = (log_prob * (log_prob - log_prob_target).detach()).mean()\n",
    "    \n",
    "\n",
    "    mean_loss = mean_lambda * mean.pow(2).mean()\n",
    "    std_loss  = std_lambda  * log_std.pow(2).mean()\n",
    "    z_loss    = z_lambda    * z.pow(2).sum(1).mean()\n",
    "\n",
    "    policy_loss += mean_loss + std_loss + z_loss\n",
    "\n",
    "    soft_q_optimizer.zero_grad()\n",
    "    q_value_loss.backward()\n",
    "    soft_q_optimizer.step()\n",
    "\n",
    "    value_optimizer.zero_grad()\n",
    "    value_loss.backward()\n",
    "    value_optimizer.step()\n",
    "\n",
    "    policy_optimizer.zero_grad()\n",
    "    policy_loss.backward()\n",
    "    policy_optimizer.step()\n",
    "    \n",
    "    \n",
    "    for target_param, param in zip(target_value_net.parameters(), value_net.parameters()):\n",
    "        target_param.data.copy_(\n",
    "            target_param.data * (1.0 - soft_tau) + param.data * soft_tau\n",
    "        )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mWARN: gym.spaces.Box autodetected dtype as <class 'numpy.float32'>. Please provide explicit dtype.\u001b[0m\n",
      "\u001b[33mWARN: gym.spaces.Box autodetected dtype as <class 'numpy.float32'>. Please provide explicit dtype.\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "env = NormalizedActions(gym.make(\"Pendulum-v0\"))\n",
    "\n",
    "action_dim = env.action_space.shape[0]\n",
    "state_dim  = env.observation_space.shape[0]\n",
    "hidden_dim = 256\n",
    "\n",
    "value_net        = ValueNetwork(state_dim, hidden_dim).to(device)\n",
    "target_value_net = ValueNetwork(state_dim, hidden_dim).to(device)\n",
    "\n",
    "soft_q_net = SoftQNetwork(state_dim, action_dim, hidden_dim).to(device)\n",
    "policy_net = PolicyNetwork(state_dim, action_dim, hidden_dim).to(device)\n",
    "\n",
    "for target_param, param in zip(target_value_net.parameters(), value_net.parameters()):\n",
    "    target_param.data.copy_(param.data)\n",
    "    \n",
    "\n",
    "value_criterion  = nn.MSELoss()\n",
    "soft_q_criterion = nn.MSELoss()\n",
    "\n",
    "value_lr  = 3e-4\n",
    "soft_q_lr = 3e-4\n",
    "policy_lr = 3e-4\n",
    "\n",
    "value_optimizer  = optim.Adam(value_net.parameters(), lr=value_lr)\n",
    "soft_q_optimizer = optim.Adam(soft_q_net.parameters(), lr=soft_q_lr)\n",
    "policy_optimizer = optim.Adam(policy_net.parameters(), lr=policy_lr)\n",
    "\n",
    "\n",
    "replay_buffer_size = 1000000\n",
    "replay_buffer = ReplayBuffer(replay_buffer_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_frames  = 40000\n",
    "max_steps   = 500\n",
    "frame_idx   = 0\n",
    "rewards     = []\n",
    "batch_size  = 128"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_frames  = 40000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAE/CAYAAABLrsQiAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzsvXeYZEd57/+pznHyzM7ubJZWcZXQIpERIIHwtS1hBFdgG2OwybZ/wLUvGNtgDLYJThgBlm0QyQRfjBAmCDBJKIAkFFdxtUGbJ890PKdD/f6oqtOne7qne2ZndmY19Xmefrb7pD7d01vfekO9r5BSYrFYLJa1S2Clb8BisVgsK4sVAovFYlnjWCGwWCyWNY4VAovFYlnjWCGwWCyWNY4VAovFYlnjWCFYBEKIM4UQ9wohMkKIP1zp+7EsL0IIKYQ4faXvw2JZLqwQLI4/AX4kpUxLKT+20jfTCiHEa/Qg9nu+bUII8SEhxIR+fEgIIXz7LxRC3C2EyOt/L1yKc9cqQoj1QoibhBBH9N9ia8P+jwohHteTikeEEK9p2B8UQnxAn58RQtwjhOhp8V59Qoiv6L/NuBDii0KILt/+C4UQtwghZoQQh4QQf+7bt1XfX9b38O+/QQjhNuwP6n2/2bA9r691sd7f7nfT8jMKIXYKIW7Wn2fOoichxNuEEHcJIRwhxA0N+9p9pqgQ4tNCiFkhxDEhxDsazv89IcQefd53hRAbfPvafabrhRCPCiGqQojXNvt7rSqklPaxwAfwA+D35tkfXAX32As8Ajzov1fgjcCjwEZgBHgIeJPeFwEOAG8HosAf6teREz13gfceWqHvrOn7AhI4fZHXXAe8BXimvs7Whv1/CZyFmpRdCkwBz/Lt/wDwQ2ALIICdQKzFe30C+B7QBXTr3+nf+/Y/BHwQCAKnAUeBX9f7tur7a/Ud3AB8oMPP/FrgCUC0+920+4zAmcDrgavUcDXnvX4DuBr4JHBDw752n+lvgFv0/5WzgWPAlXrfZcAocK7+bX8S+Ekn/4/0/rcCLwLuAl67Er/nBf1OV/oGTrWH/sFWgCKQBc7Q/0k+CXwbyAGXA/8LuAeYBQ4C72vyA/1dvW8KeBPwdOB+YBr4eMP7vg54WB97M7ClzX1+CjUA/Zh6IbgNeIPv9euBO/TzFwOHzX9gve1J33+ORZ/bwfcq9X+ex4F9ettZwPeBSf2f7pV6+zb9HQX0638FRn3X+jzw/+nnv6u/twywF3ij77jLgEPA/9WDwOf19j9GDZJH9Pe+aCHwvVeIJkLQ5LibgHfq5736N3Zah+/xHeAtvtdvBW72vc4D5/he/yfw7obf5FIIwY+A93b4m+voMwKn00QIfPs/wMKF4AjwYt/rvwK+rJ9/FLjOt2+DvtZp7T5Tw3v8jFNACKxraIFIKV+ImkW8TUqZklI+pne9GjXbSqP++DngNUAPShTeLIS4uuFylwI7gP8N/CPwHpSInAu8UgjxfAAhxFXAn6JmP4P6/b/U6h6FEJcAu1Bi0Mi5wH2+1/fpbWbf/VL/gjX3N+xf7LmdcDXqOzlHCJFEicB/AEPAtcAnhBDnSCn3oQT2In3e84CsEOJs/fr5wE/081HgV1Gz5N8F/kEI8TTfew4DfajZ6BuEEFcC/we4AvW3udx/g0KIVwsh7l/AZ+oYIUQcNRnYrTedB5SBa7Tr4jEhxFvnucR1wK8KIXqFEL3Ay1HiYPhH4DVCiLAQ4kyUlfKDhmsc0G6jzwghBhr2vUUIMandfi9v8Rm2oP4en/Ntnu93s9DPuBjmfCb9/ayf575AWSeNz3fqf+f7TKccVgiWjm9IKW+VUlallEUp5Y+llA/o1/ejBu7nN5zzV/rY76GE40tSylEp5WHUYG8GujcBfyOlfFhKWQb+GrhQ/6erQ/ttP4ESqmqT+0wBM77XM0BK+zcb95n96SU4txP+Rko5KaUsoAbv/VLKz0gpy1LKe4CvAa/Qx/4EeL4QYli//n/69TbUoH8fgJTyW1LKJ6TiJyjXyXN971lFzV4d/b6vBD4jpXxQSpkD3ue/QSnlf0gpz1/AZ1oIn9L3fbN+vRHl4jkDZQVdA7xPCHFFi/N/iXJjTOhHBfVbMPy3vkYB5Tb8dynlnXrfOEqEtgAXo/5uX/Sd+zGUMA4Bfw7cIIR4dpN7eA1wixZrw3y/m4V+xoUw32dK+e7Ff1/m9/pd1GTsfC3Qf4GyCBIdfKZTDisES8dB/wshxKVCiB8JIcaEEDOowbxxhnXc97zQ5LX5sW4B/kkIMS2EmEa5SgTKN9nIW1Az8zta3GcWNVAauoCsnsk37jP7M0twbif4v8MtwKXmM+vP/ZuoGTwoIbgMNfv8KcoF9nz9uMWIoBDipUKIO/RMdhr4Fer/DmNSyqLv9YaG+zjQ6c0LIZ7rC0rubn9G3bkfQc02X+mzqgr63/dLKQt6QvFl/Rma8VXgMdRg1oXy039BX78PNbi9H4gBm4CXCCHeAiClzEop79Kiexx4G/BiIURa7/+llHJC7/82akD9jSb38Brgsw3b5vvdLPQzdkybz5T13Yv/vjL63B8A70VNPvbrRwblSmz3mU45rBAsHY0/gP9A+Xs3SSm7UbO9xc4WDqJ82z2+R1xKeVuTY18EvEyb2ceAZwF/J4T4uN6/G7jAd/wF1FwRu4HzG2Y15zfsX+y5neD/Dg+ignP+z5ySUr5Z7/8JamZ/mX7+M+DZ+NxCQogo6j/yR4F1UsoeVBzHf4+Nf7ejqEHSsLnjm5fyFn2PKSllx24CIcRfAi9F+atnfbuMC8p/j/MNNBcC/yKlzEkps6jfnBlQtwMVKeXn9MB4iPkHXPM+rcYIScPvWVsIG1DWmZ/5fjcL/YwngveZpJRTqL91q/tCSnmdlHKHlHId6ncUQiVfwPyf6dTjZAYknioP5gZgb6AhkIbyTf+Ofn6Jfv0F/XorDUEs1EzjMt/rLwB/pp+/DPUDPFe/7gZe0eLeelCzZvO4DXgH0K33vwkVPB1B/afdzdzMnz9CZf68jfqsoUWf28F3WheQRc1qDwC/DYT14+nA2b5jjqBiBZv06zv166f7rlFBiYNADbZ587dCB4sb7uOlqMDxOSg3wBca720Rv5cYkNTXORNf1g/wblSAfLjFuT8F/kV/p2fr39GLWhz7I+Cfgbh+fAK4Te/rQgXYX40a3IeB24G/1vsv1fcWAPqBr6BSpM21r0FZqAFUYkAG3+9VH3M98Lkm99Xyd9PuM+q/W0z/PaR+HvWdG9Lb/gaVJBBD/7/q4DP9LWrS0ItKTDhKLTEihrLQBGoy8GPzXXX4mSL6GrcCv6+fB1Z67Gr5G13pGzgVH3QmBNegBrIMyjf7cRYpBPr1bwMPUMtC+vQi71UAH0a5lyb1c3+mz0XA3SiT/ZfARUt07p8C35nnPucMtvo/8beAMZTP+4fAhb79X0JnGOnXH9Xfd9C37a0ol9u0Hii+zDxCoLe/CyUGc7KGUO6p3Qv8vcjGR8M+B+VqMI8/9e0fQbl0sszNeqq7F5SP/Zv6u5rU5+3w7X8hSixn9Of7VyCh970K2IeKVR1FBXuHfefeos+bRcUxrm34jDH9Hc8RqQ5+N/N9xq1Nvr/9vv3va7L/fR1+pijwaf2ZjgPv8O3rQVkrOf1d/U3D76rdZ/pxk/u6rPG7WS0Pk+drsVgsljWKjRFYLBbLGscKgcVisaxxrBBYLBbLGscKgcVisaxxrBBYLBbLGie00jdwogwMDMitW7eu9G1YLBbLquPuu+8el1IOtjvulBeCrVu3ctddd630bVgsFsuqQwjRUYkU6xqyWCyWNY4VAovFYlnjWCGwWCyWNY4VAovFYlnjWCGwWCyWNY4VAovFYlnjWCGwWCyWNc6qEwIhxJVCiEeFEHuEEO9a6fuxWCyWpzqrSgh04/XrUF2izgFeJYQ4Z2XvymKxWJ7arCohQLV03COl3CuldFHdpK5a4XuyWCwLoFqVHJzMtz3mwcMzJ/Q++8dzHJjIdXTsXfsnWWwTrjv2TnDL42OMZZw5+6SU5N3yoq5r2DOaoeBWvNeV6olfc6GsNiEYQbVhNBzS2+oQQrxBCHGXEOKusbGxk3ZzFoulPTfvPsZlH/0xo7PFlsf8zyOj/Oo//6ytYMzHn934IH9244Ntj3v0WIZrPnU7P318fMHvcWS6wLXX38Fv//svePW/3jFn/z98/zHOfe/NvPSfbuHho7Mtr1OpSt78hbu5+8Bk3fZiqcL/+tjP+Pwd+71tX7jjAM/78I+oVE9e98jVJgQdIaW8Xkq5S0q5a3CwbT0li8VyEtk/kadSlRyaLrQ8xgjAdL606PeZLZaYyrttj5spqPc4PNX6flphrj/SE+fgVH6OVXFgMk8iHOTho7P89LHWk9Jsscx3HjzGHXvrhWA86+CUqxzy3dujxzOMZ10mc+0/21Kx2oTgMLDJ93qj3maxWE4RJrLKhTLexJViGNPHOOVKy2Pa4ZSq5N3255v3aObaaYe5/vbBJMVSlVzD++WcCpv6EsTCAcazra/vVNR5brlat3086+p/a+eOzjqLvt/FstqE4E5ghxBimxAiAlwL3LTC92SxWBaAGdTMINf0GD3INQ6MC6FYrpB3OhCCUrXuvhZCzlG++s19CaAmcoa8WyYVDTGQis77ec3ndBo+rxnsxzOub5tyqY0t4n4Xy6oSAillGXgbcDPwMPBVKeXulb0ri8WyECa0S6Nx0PRTswgWLwROqUqug6CqeY/FzLBzWmi29CshaBSTnFshEQ3Rn4rOKzQ1IagXLnOOf9A39zlfjGWpWXX9CKSU3wa+vdL3YbFYFkczd8fcY5ZACMoV8m4FKSVCiHmPa3c/rTBCYyyCxll/3imzoTtGJCg4PN164HYrzS2C8Uy9G01K6YnCmrUILBbLqc9EB64hM+s9oRhBuUqlKtuKiWcRLGJgzWvX0Jb+JDBXTPJuhUTEuIY6sAhKDa4hfU7GKVMsVZjKlyhVVEDaxApOBqvOIrBYLKcu1ar0sl1aDYzVqmRCi8SJWQTq3LxbIRYOtj6upC2CxbiGdHB4k7EIMm7D/jLJaJCuWJjJnEu1KgkE5lonRgiMZWDwf0fjWYesU3N1WYvAYrGckswUSpR1/nsrIZj2HbPYYHGpUvXy7HPO/HECIxg5t7LghVp5t0woIEhGgnTHw0zkGiwCp0IyGmIgFaFSlS3TWWsWQUOMwCcs41nXs5QiwQBjJ9EisEJgsViWDDNQdsfDLV1DfoFYrEXgP69dCqn/2MYZfTtyToVEJIgQgoFUpO7e3XIVt1IlGQnSn4qq67f4zE6rGEHWYVNfXN+b47mDzhxOW4vAYrGcmpiB8MzhNDOFUtMZvz97Z7EWgX9m3S5zyB+HGMsuLBMn75ZJRpUHvb8hRdSUhTAxAmhtBZnYQGNMZCzjcPZwl3fuqP5uzt3QdVKzhqwQWCyWJcP4/s8eTgM0XR1bbxEsLlhcZxG0WUvgD9COLdQicJVFADDYEBDOagFKRoMMpiNAayFoljVULFXIOGXOWu8XgiLJSJAt/UlybqWt22upsEJgsViWDOMaOtM3y23EbxEshWuovUXgE4IFultyjt8iiNQFnE1GUb1F0C5G4HNT6XsZ6YmRjoUYyziMZRwG01EG0/NbGEuNFQKLxbJkjGcchIAd61JA84F3LOsQCQZIRUOLdg0Vfa6hQpsYgVuuMpCKIsTCM4fyTs0iGEhFmS2WvXs2GUXJqAokh4OitUXQJGvIiMZgOqqtDZfRjMNQOsaQFoLRk1RmwgqBxWJZMsZzLn2JiDeQTTSZIY9lHAZSEaKhwJK4hjqJESSjQfoSkYVbBLqEBODN+o3V47cIhBD0J6MthcbVn7MuXqGPHUhFGUhFGctqi6CrZhGcrHpDVggsFsuSMZF16E9F5g2ejmddBtNRJQSlEw8Wt40RlKtEQwG16GuegfW2PeOU9Yz9voPTui+AWjAGyjUENXEzFkHK7zpqFyNo4hoaSKmB/+hMgeOzRQZT0ZpFcJICxlYILBbLgqlWJdUm9fInsi4DqSjJaIh4OMixmbkDmbIIokRCgTkLrAxZpzyvy6fRInjg0AzfvO8IByfzvPOr9/HKT93Oh777CHm3rIUgyGA62tIi2D+e49X/9nNu3n2cJyfyXHXdrXznwaM6RlBzDQH8510H+dEjo96aBL/ryLh7brrvCO/4yr3e9ZsVnTOipIQzwsHJAnm3wjO299ObiNCbCPPpW/ezb7yz5jsnghUCi8WyYK7+xK2c897vcu31t9e5OyZyLn1JNXMe6Y1zw237ufIff1q3YnY862iLINjSInjLF3/Je258oOX7N64j+Kv/fog/+NI9PPfDP+Kb9x8h45T55I+f4Od7J3HKFaKhAIPpaMuyDWYh2NGZAoemVa+EQ1OFOotgfXcMgM/efoD/85/3eQXpkj7XkZnl/+TRMb5x3xFPLJsVnRvPOnTHw0RDQS47c4hnbu/nP9/0TK7cOUwgIPj0a59Opljimk/e1lRQlxJbYsJisSyIalWy+8gsyUiQO/ZOcmS6yLYBVYvHZL0AfP71l3D9T/fymVv3s28sx3kbu6lUJRPZ9hbBocm851dvhj9YnHPKHJstcvGWXp5z+gBXXzRCpVrl8r9XAuSUqkTDAYa6ooxlnKZF6oz1MZZ1vFn98dmiKiGhZ/wbeuJ85Q3P4MZ7j/ClXzzJdEEdV7MIIkzkXKRUK4zNSuP+VLRuQZl5/7GsipUAvOCsIV5w1lDdPV20uZevvflZ3HjvEdZ1Rdv9WU4IaxFYLJYFMV0oUalKLtjUA9Rm03m3TNYpM5RWM+f13XGuulB1mjULuabyLlVJLUbQYrDPOuV5Vwwbi0AIJQSjmSJP29zD2684g20DSW8W73cNDaVjuJVq065o5r3GM67nsjk4mUdKSERr8+VLt/dz/sZuAA6MK8vBvFdfMoJbrpJ1yt76CVOS21gEUlIrwZFxPXdTK7YPpnjHFWfMW111KbBCYLFYFoSpLrpjSC0am9ZCYDJcjEXgf97YdWsgFSUabh0szjnleRdTGQHpiYc5PutQLFXr3te4a3JOxXMNmVl1s5TMfKlmEZg4gvHNG4vAYK6zbzxHLBwgqIvMmTITkznXE8dmDXiMiI1nHQbSyzvT7xQrBBaLZUGYgdKsFZjKqRm2GWCHfIObcX14nbiyNbGIBJu7hqpVSc6tzB8s1gLSm4xwYCKn3zfm7TfumpyjLIJIKODtP94kE6egA7/jGcdnEag+wslovQfdXGfveM7LGALo17GRiZzLpOnJ0GARqHuvic5gG4vgZGGFwGKxLAjjQ98xpIVgHosgGgrSkwh7IlGzCCItg8VmXUBjf2A/Zlbdl4hwRAdS/QIUDgaIhALk3IqKEbSzCIxrKOt4YmVEyrh+DEO+Vb/+fSZIfnymSMapCYv/Wubei6UKmWK57rtaSawQWCyWBWEGt20DSQICz+duct6HGga3wVS0uUXQIlhssnHmKxltgsW9evAFGGoIqCYjQR0jqHgxAoDRzFyLwAjBRM6dIxQmfdTQn4piWg4kfG4jIwR7RrPeNk9UGlxDtTUEtftfSawQWCyWBTGedQgGBL2JCD2JSM0iyDqE9HY/Q121/P2xjEM0pMpLqAVlc2f9JtW0VJEtS1A45apXpsIw6HMNgZrJqxiBsgjikSDpWKhpCqlxQ1WqkifGsoSDou46foIB4QV5/W4js+Bsz1hNCMziM//ncMtVz6pqFyw+WVghsFgsC2I869CfjBAICHoSYZ9FoNJCGzt0Daai3izcrCoWQhAJBZoWnfMHiVvFCUwA2MzII6EAXbH6ATsZVRaBW1bpo6CslfksAoBiqeoFws11GlnXpUTHbxEkImoRXVOLoM41VPGsKusaslgspyRm9TBAr88iGM04c9wzoAY7k79vVhWDih80m/H7F5+1qiPk6MHdzMiHtLj4SURCzBZVN7RoSA3Y67piHG9mEZTq3+dsXRoaIBmZu9zKuL8a9/UlIzyhLYJ1XbUFZk6Da2jMV15iNWCFwGKxLAh/2mNvIsyUtgjGMs2zYIbSMYqlKhmn7K0qBlT6aBshaBUnUAHgoDcjb4xLgJrJm4ymaKi9ReDXkXM2+IQg2kQIjEXQYC0MpCIUdQB8x1DacwG55SoxbZU4pWpdeYnVgBUCi8WyIMazrhfk7ElEmOnAIgC8evtmFmzSR6Wsr1nkdw3lWhSUK5YryiKIGIsgNueYRCTkWStGCIxF0PieebfC+q7aNc5cl24aEDYY4UlF51oEhtOHUoxn1Xu55SrpWBjQrqGsQ1cs5FkqK40VAovF0jFSSl0aod4iqFQlk7nmFoERgmMzRSbzbp1FAHOb0+TqLIIWMQJjEegZeTNfezIS9Fb2RsO149xyldlCvaVRcCus644RCRrBiNKfihIMCE9E/NRiBI1CoO6jKxZifXcMp1wl51ZwK1XSOoZhXEOrZTEZWCGwWCwLIOOUdaOXmkVQKFU4Ml1QpSO65s7Mzez5kWMZpIRBfa4ZdBuFINOJa6ghWNzMNZTwNb7xWwQAxxvcQ3m3TDIS8lw1A7oUtGlc3+ozNa46Nt9LXzJSa2ifceosArdcZTzjrprFZLCMQiCE+IgQ4hEhxP1CiK8LIXr09q1CiIIQ4l79+JTvnIuFEA8IIfYIIT4mlrvAhsViWRATDWmPJlX0seMZgHktgoeOzNa9NrP0xoBxnWuoZdZQVQuBdg01cUn5B2njgjED+IGJfN2xebdCPBJkIBUlFBB0x8MMpqNNA8X+90u0cA31JiOeKIxnlRB0eRZBZVWVl4DltQi+D+yUUp4PPAa827fvCSnlhfrxJt/2TwK/D+zQjyuX8f4sFssCGW/IdulNqFnuY8dVpkyzAdm0cbznySnAJwQhYxHUD/b+uEC+Rb0hlTUU9Hz0rWIEBvNe52zoYrgrxnu/8SD3HZzmpvuO4JQrFEqqJeVgOuqlwF5xzjpefO66pu+/uS9BIhJkc1+ibrsRgr6EvzmPi1upevdqXEOrySJYtjLUUsrv+V7eAVwz3/FCiPVAl5TyDv36c8DVwHeW6x4tFsvCGM/UC0GPtgjuPzQN1FwvfoQQDKVj7B3PkYgE2dKvSlabwdlYBF+/5xDd8TBZR5V+zrkV8m6Fv/72wyQjIf7o8h3eNZ1ShVg6yq6tvfzxS87kWaf3z3lff/5/RL9XOhbm335nF9d86jauuu5WAD7xm0/TfQeC/PoFGzg8rWoM/ealW1p+Dz2JCHf92eXEw42rjmsWgb8BvXINqeF2tlAiUyyvmlXFcPL6EbwO+Irv9TYhxD3ALPBnUspbgBHgkO+YQ3qbxWJZBfzp1x/ga3er/6LDuklLb1JZBP/z8Chb+hNs6J4rBAAfueZ8xnMul505SJf2ldcsAiUEH/ufPazritKbUINobiJP3i3zg4eO050I1wuBtgiioSBvfcHpTd+zmUUAsHOkm0+/9un88sAUH/3eY4zOFim4FeLhEC86u7kF0O76hn4dLO5LRujR1tJ03q2LERyeVvGJ1bKYDE5QCIQQPwCGm+x6j5TyG/qY9wBl4It631Fgs5RyQghxMXCjEOLcBb7vG4A3AGzevHmxt2+xWBbAN+45zM6Rbt5xxRk1X7i2CNxKlSt3Dresm/+s0wfmbIs0CMF41qFSlYSDAXoSESLTRXJuhbGMQ6Uh3dMpVZpm8/jxp3ZGG2buzzptgEu39fN333+MiZxL3i03TRNdKP7vJRoKEg8HmcqXcCtVktEQQuBZHKtlMRmcoBBIKS+fb78Q4rXArwIvkjpxV0rpAI5+frcQ4gngDOAwsNF3+ka9rdn7Xg9cD7Br1665jVMtFsuSUq1K8qUKzz59gGf7BnUz6wV46c71C7qmCeC6vmqcxVKF3mSEdDREIhpkKufWZREZTLB4PhJ1weK5x5p6SUdnilQlxJdACDb0xHn1pZu5/GzVbawnEfbiKtFQgGgowOEpFaheTUKwnFlDVwJ/Avy6lDLv2z4ohAjq59tRQeG9UsqjwKwQ4hk6W+g1wDeW6/4sFkvnFMsV1a2rYbA0q3s3dMe4QHfu6pSIL1hsOnqVKpIDEzmS0SDJSMjL7sk45bqgslOuEgvPP3D7VwS3Eo3+ZIRDemBu9PcvhmBA8NcvO48d61Stou542Ku8GgkGiIaCNYvgqeIaasPHgSjwfW0u3qEzhJ4HvF8IUQKqwJuklJP6nLcANwBxVJDYBootllWA16i9yaz50m197Nrat+B2il6MoFQrywyqrHUyGiIeCfLkZC3NcypXYrhbvX+xA9dQnUXQYpDvS0Y4NFWYc/xS0R2v9WKIhFSPhJmCKnuxJoLFUsqmERwp5deAr7XYdxewc7nuyWKxLA6zsKtZgPQzv3vJoq7puYYqVW99giEVDZGMBL0CbgATOYfh7hjlSrWukFwrOrIIUhHuOqDSWpfCNdRIdzzMI8fUGouIdg0Bq6q8BJy8rCGLxXIKY0o9NCvJvFj8rqHG/sTJaIhEJIQ/RmwKyJmSzqZERSvaxQhAZflUdDP5ZiJ3ovQkwp4FoFxD6j5Wk1sIbIkJi8XSAfNZBIvFv47AVOk021LR0BzRmcjpks66umesjWsoWZc+2to1ZFgu15BBWQTqPVZToBisEFgslg7wYgTLYhFUmcg6xMNBtg2oxWapaIi4HsiDugyoCSibdNNWfn+DKUgnBHUdx/z4y0Avh2uox9etLRIKeFbMalpDAFYILBZLByynRWCCxQPpCCM9cUC5hkxgenNfgoCoCYHpV9wuWBwJBgjp6qGtAtnLbRF0zbEItBCsMovAxggsFktbjEWwlIOlsQjcSpWJnEt/MsoGLQSpaLBWUC4dZbZQ8kpKexZBm2CrEKJl9VCDWQkMkAgvQ4zAJwTRoN81tHoyhsBaBBaLpQPyJSMESzdYemWoSxWvYc0Gv0WgXTsD6Sh9yQiTWSME6l5ibYLF5jrzWQ7L7RpqjBEY8bMxAovFcsphqoAuZYxACOW2cbRFMJCKMNKrhCAdC3sD82BKC8ECLQJQFkxkHiFYbteQf+V1nWvIxggsFsupRk739I0tce57JBSg6KqVxQOpKFecvY4//9XSSeSRAAAgAElEQVRzOG+k28v6GTQWgW47aVbqpmLtrZN2FkFvIuL1Kl6KlcWNnCpZQzZGYLFY2pJ3yiTCQQKBpe0VFQ0FOT6ris31pyLEI0Fe/5xtQG2GPpCK1FkEt+4ZJxkJcs76rpbXNSQiQcqV1uXITL2hvFte8s8G0BP3ZQ0Fa1lDq20dgRUCi8XSlpxbmdONaymIhgIcnWlejdOsDB5IRelPRpjKu1Sqkp/tGeeZp/XP6/IxXH72OmaLzZvbGPzuoaUmHVMVR6Wsdw31L+N7LgYrBBaLpS1LVaa5kWgowN6xHFAfuAWVNhoOCk4fSnFwMo+UqgHOgYk8v/usrR1d//eeu73tMX3JCIUWLTFPlEBA0BVTq4sjoQAXbe7lwES+bcG8k40VAovF0hbVwWvph4uLt/Ry472HiYeDnDaYqtu3c6SbB//yJURDQfq0tfCVOw8C8NwzBpfsHrYPJJFy+arZd8eVEESDqgPar1+wYdnea7FYIbBYLG3Ju+WmlUdPlI+84gI+fM35VKqSUHCuq8cEV7cPJAkI+PKdB9nQHWO7XoG8FPzFr51DaZ44wonSkwjz5CQdubJWCisEFstJpFSpUqnKVecaaEfOqdStkl1KhBCEWpSAMOwc6eaH77yML935JOes71pwyev5WA5Lx4/JHLJCYLFYAPjb7zzCPU9O8V9vefZK30pH3LpnnK0DSfJumfUt+hGfLLYOJHn3S89e0XtYDN3xMMGA8GomrUasEFgsJ5EnJ/M8fjzb/sBVgJSS3//cXbzsohFyzvLECNYCPYmwt4p6tWL/shbLSSTvlsk4qjfvancPTedL5N0KB6cKFEqVZckaWgtc+/TNnDXcfs3DSmKFwGI5iZjibWMZh019iRW+m/k5NlsE4NBUnpxT9so6WxbGzpFudo4srJ/zyWZ12ysWy1MMU87Z36N3tVITggJOuVrX6MXy1MIKgcVyEjEWwXhDj97VyLEZJQSuLvJmXUNPXawQWCwnkVPKItBCYEguQ4kJy+rACoHFchLJubUYwWrn+Gy9EFiL4KmLFQKL5SRRqlQ9N8upYBEcnSmytb8W0Lbpo09drBBYLCeJvK+w2akgBMdni5w+lKZL1/1fjhITltWBFQKL5SRh4gMA45nVESwulio8eizTdN+x2SLD3VE29iqrYDnKUFtWB8smBEKI9wkhDgsh7tWPX/Hte7cQYo8Q4lEhxEt826/U2/YIId61XPdmsawEJmMoGBCrxiK47kd7+LV//lmdSIESiOl8ieGumNc+0loET12WW+L/QUr5Uf8GIcQ5wLXAucAG4AdCiDP07uuAK4BDwJ1CiJuklA8t8z1aLCeFnO77u6EntmqCxd998BhupcrByQJnDqe97SZjaF1XjI29ynqxFsFTl5VwDV0FfFlK6Ugp9wF7gEv0Y4+Ucq+U0gW+rI+1WJ4S5PSse0tf0iszsZLsH8/x+Kiqe3RwMu9tPzxd8Lav746zYyhNJBio679reWqx3BL/NiHEa4C7gHdKKaeAEeAO3zGH9DaAgw3bL13m+7NYThp57Rra0p/gZ3tUwNj430/qfbhlnhjN8fN9E962g1M1Ibj6ulu9/sDD3VEu3d7HM0/rJ2UtgqcsJ2QRCCF+IIR4sMnjKuCTwGnAhcBR4O+W4H7N+75BCHGXEOKusbGxpbqsZY1y655x7tg70f7AE8RYBFv7VVOVpV5dfMvjY8zkS3O2//CR44xmamsCPnf7AX7t4z/jo997lDPXpYmHgxycVH2DK1XJWMYhFBCkYyE29MQJBwNsW8JGMJbVxwkJgZTycinlziaPb0gpj0spK1LKKvCvKNcPwGFgk+8yG/W2Vtubve/1UspdUspdg4NL17LO8tRHSsm9B6frtn3k5kf58HcfWfb3Numjm3Vu/vgSxgkyxRK/8+lf8IWfH6jbPpVzed0Nd/H522vbD08VvLLIV180wqa+OIe0RWDE6h1XnMFt73qhXTuwRljOrKH1vpcvAx7Uz28CrhVCRIUQ24AdwC+AO4EdQohtQogIKqB803Ldn2VtcvveCa6+7lYeOTbrbcs5ZY7PLn/w1gSLdwyp3rwHfH75E+X4bJGqhKMzhbrt9x1SondkumYRjGcdNvcnuP+9L+GNz9vOpt4EB6cKdfeYjoVJx2xMYK2wnHL/YSHEhYAE9gNvBJBS7hZCfBV4CCgDb5VSVgCEEG8DbgaCwKellLuX8f4sa5Bp7ToZz7gwrLbl3YoaSKuSwDJ2kfIsgr4E67tj3H9ous0ZnWOEbLRB0B44NKP314RgIuvSn4x4rRM39sb5xb5JpJSeECRtyek1xbIJgZTyt+fZ90Hgg022fxv49nLdk8XilNVgnPPlzWedMuWqZDLvMpCKLtt759wy0VCAUDDA+Ru7ue/g0gmBiQGMNrib7tNCcMwnBOM5h7PX1xqlbOpLkHHKzBRKZHVA2waG1xZ2ZbFlTVEsqVo//gVU5nljtc2lJu9UvAqeF2zqYf9Enun84gLG03mXh4/W3Fs1i6D+Mxirw28RjGccBpIR77XJXDo4WfBZBFYI1hJWCCxrCkfn7puZr1uuUqpIYG61zaUm55a9Cp4XbOwB4H49Y18oH//hHl75qduRUt27cQmNZR1v27GZIqMZh+GuGJlimbxbxi1XmS2W6fdZPpv61Mrhg1N5sloIrEWwtrBCYFlTFHX1z7we8PyWwbFlFoK8U/G6fJ23UbUuXKx7aP9EnoxT9lxBx7VrqFSRTOk4iLEGrjhnHaCEwawPGKgTAmURmJaUYC2CtYYVAsuawtGuITPg5XwVQY8vs2so59b6/nbFwpw2mPSyehbKkWmV5XNgQmUejfmCxCZe8NDRWQICXnCWSrE+Nlv0ahz1p2quoa5YmGQkyNGZog0Wr1GsEFjWFEUvWKz+NZYBnASLwK3U9f09d0M3j7So/NkOkyZ6YCIHKItgQ3dMPTduooxDbyLCFr2A7bhPCAZ8QgDQk4jYYPEaxgqBZU3hNASLs3VCsLxrCXJOua7LV38q4qWzLoSCW/HcPwcn80gpGZ11OHdEuZtMwHgq79KTCLOuSwnEsRmHCb2auT9Znx3VmwwznS+Rc8oEBMTD1iJYS1ghsKwpTPqomfma3P6uWGjZXUN5t1Lne++Oh1XqaqW6oOsc8S0aOzCpYgWFUoXzjBDouMFUrkRfMkIqGiIVDXF8tshETlsE6Xoh6IlHmM67ZJ0yyUgIIZZvPYVl9WGFwLKm8NJHTYxA/7t9MOUFXJeLvFtvEZhqnrPFcqtTmmLiA/FwkAMTeS9jaEt/gnQs1GARKBfQuq6odg25REOBOb0FuhNhpgsl8m7ZBorXIFYILGuKmkVgsobU69MGU0znS8taGjrnzLUIAGYKys1TrUr+/nuPzikT0YgRgl1be3lyMu8N/EPpGEPpaM0iyLv0aSEY7o55weKBVHTOjL83YVxDFRsoXoNYIbCsKWoLyupXGG8frAVUl4NKVVIoVZpaBEYIDkzm+dgP9/CDh0fnvdaR6SJCwCVb+5jMuTwxpnoHDHVFGUrHGM2otQRTuRK9SWMRxDg+U2Qi684JFEPNNZRxyjZQvAaxQmBZUzSWmDA9ArYPGCFYnoCxCU77s4YahWBS+++L7vxWyZHpAkPpKKfp4nV3HZgC1GA/1BVlNFMk51ZwK1V6E+o9NnTHOZ5x2DOarVtMZuhJhKlKlUJrXUNrDysEljWFU65fR5D12keq1bWdlHyoVOWCArx7RrN88edPAszrGjL9Cdq5p47OFNnQE2ezXgj23/cfZbgrRioaUjP/WYcpvXDMWASv3LWJeDjI4ekC/ckmFoF2IR2eLlghWINYIbCsOHtGs7z7vx6gUpV124/OFPjQdx+h2rD9RDAlJvJe1lCZeDhInx4cpwvt0znf/V/386Yv/LLj9/yH7z/G335H9TvwL+SaaxGowbvQRgiOTBfY0BNn60CSZCTImevSfP71qt3HSE8ct1zlseNqfUKvHuA39yf44Mt2AjCYbmIR6HvJWtfQmsT+xS0rzk8fG+NLv3iSP3zR6azvjnvbb37wGJ/88RO86umbvWYuJ4pnEbhlVXbZVcHRbu1Cadbhq5GHj2a8wfvwdAGnVGH7YKrl8WNZh/NGunnfr5/j1RgC6DJZQ/paE3qx13xCIKXkyEyBF509RCoa4sd//AJ6EmHCutGMqRtkqo72JWs9Ba66cIRwMMBFm3vmXLcnUTsuEbHB4rWGtQgsK44ZnPMNvvEJPUPOl8qUKtU5ncUWg3G7VKUKHOedMolIiHQ0RDAgmC60dw2NZRxv0H7fTbv5oy/fO+/xUzmXkZ44F2/pIxSs/ZeLhYNEQwFPVMznNQHtZswWyhRLVW+R2GA66okAwCZdSfQBXbrCWASGXzlvfZ3YGnp8x1mLYO1hhcCy4pjBudBKCNwK33nwGFdfdysHF9jV6wP//RDX//QJ77URHVBWQU4v8hJC0B0Pe4NyK6pVyXjWIedWKLgVDk8VvHTOf7tlL3/13w/NOWcq73q++ka642HPCpnoIEaQcUreec0wJaVNVdNGIWiF3yKwMYK1hxUCy4pjBudGl8ikHhgLboVJPQPfo1MlO+WHj45yy+Pj3utiqeItpso5ZXJO2XvdEw+3LfkwXShR1jGLiZzDaMZhMu9SqlS5efcxvnnfkbrjq1VVDdTvovHjFx8vRjBP1lDWayXZfLCOR4IMpKJM5FwCouZ+aodfWKwQrD2sEFhWHJPS2egaMgNjzil7ReIOjOfaXm8mX/LSNfNOpe66TrlKnw7Y5pwKObdCQg983Yn2FsGYrwPY8VmHyZyDlGo2f3zWYSzr4PqsjkyxTKUq6Us273zmFwJTEM4UxmtGtti+TPRmHSfojocJdth6MxwMkNbXTNkFZWsOKwSWFcf4xAtufamF8VwteGoG9k4avv/OZ37BX96kXDQ5tzxXCLS7JOeWyfssgu4mFsFMvsRb/+OXniiZwRrgseMZTELTaKbIsdkiUtYvSpvU6ahLZRFkOmgcY/oLtHJHtcIEzK1FsPawQmBZcdpZBHm3Qk6ne5r6+60oV6rsPjLDkZkCUkrybk1ESpWqnp0bi0CJREIv8uppEiO4c/8k37r/KPceVIu2/BbBI75WkY8fz3qWwOHpWokI8xla+eqNEEgpvWPnixEYi6CVawhqAeO+DuMDhh4rBGsWKwSWFadZ1lC5UvVm5/7B3NTfb8WhqQKliiTrlHHKauA31zXvY9w0ebdCzi17rpCeRGTOgjJT9yejB2C/EDx8tNZL4IHDM3POAbyFXX0tZudd8TCzhRKzhbIXe5gvfbTWSrK179+kkPYsUAiMWNmsobWH/YtbVhynSdbQpG9ALri1GMHByQKVqmzp+zZ1d7LFmkvIVBo1M22zqCvrlMk7vhhBPMys9umb6x/RpamNRTKWdYiGAgQDgoeP1SwCvxAcmZ7rGprPIlAtJ9U5wYCYN33UWASpTiyCFu6oVpiAsb8MhmVtYC0Cy4rTzCIwbhKz3QzmbqU6b3VOIwQmIwggX6ogpfRZBGpQnsmXcCvVWtZQon6BF8BR7ebJ6rTNsYzDYDpKfyriWQmJSJDdR5QQCFHvGmpnEZjBd58Ogq/vjs1rEWScMkJAYp7GMYuNEViLYO1ihcCyImSdMj/TaZ1e17BSLVhsUkfBuHAqhPQs/cl54gR7x3Le9U1hOakXjxnLwwzKZhbuxQi0EPjLTBiLwDSy8YRAu5e6YiFGeuLeLH7HUMoTD1AWQSQUaLlat1EIRnri8xadyxbLpCIhAvNkA63vjnHuhi4u2jR3BfF81GIENmtorWGFwLIifO3uQ/z2p3/OTL7kBYv9A+BEzu8aUjGC03W1zf3zCIHnGvJZBKAyhMxgnY6GCAWE5+83A19j7R+o+fuzvhjBYCrqlXIeTEcZ6lKiMJBS/YH9rqGpnOoJ0Krjl3lPI2AjvfH500ed0rxuIYBQMMC3/vC5XLlz/bzHNXLR5h7OG+luuVjN8tRl2YRACPEVIcS9+rFfCHGv3r5VCFHw7fuU75yLhRAPCCH2CCE+Jmy/vKcsEzkXKdVK2cYeAVCru9MdD+s0zwrbB5NEggH+8+6D/P33Hm1ajM4MqFVZW6kLSkyM4MTCQZLREGP6PYxF0B3Xhee0X79alRzzYgRaCLL1FsFQOsagLuu8rivGhu6Yt9IYYNLXE6AZJmVz77gSsJGeOKWKpNSiuulyFoV74Vnr+OYfPKeuDIZlbbBsf3Ep5f+WUl4opbwQ+BrwX77dT5h9Uso3+bZ/Evh9YId+XLlc92dZWYwf3j9A50v1MQIhtM9cZ/eko2GeeVo/9x+a4WM/3MPehsVlUzmXiZzrlWce9WX4+C2CaFi1ajQtHs3Aalwj/to/pYoSm6yr6h1N5V0GUlEv4KwsAlX3Z7grxoaeOBmnzGxRXWMq784btDWz791HZknHQnTF1Gt/CqlTrvBnNz7A48czZIrlthaBxbJQll369az+lcCX2hy3HuiSUt4hpZTA54Crl/v+LEvDg4dnuO5Hezo+3gyUebdSKzHR4BrqS0RIx0I6WFwhEQ3y2dddwr/81sX63PoFaGZWff7G+ibuoLJ+jOBEQ0ES0RBP6sVpCV+JCcBLW/UHpbPFMpPailHBYmMRRGsWQXfM62twVLuHpnLuvPV+NvUmOH9jN8PdMV7+tI3E9L34M4c+c+t+vnDHk3zvoeO2TLRlWTgZNuBzgeNSysd927YJIe4RQvxECPFcvW0EOOQ75pDeZlnljGcdXv/ZO/nIzY923PPXZNzk3Yp3jn9gn8y59CUjxCMh8iVlEZi0RjNwNy5AOzipBu5zN2gh8K3wLbgVb3CNhQOs745RqUquunAD52nh6G4QAuPr74qFyDllL6Yw0BAjMPX9jUUAteDvZN5t2gjGEI8Eueltz+GH77yM9/36ucTDRghqwemP/3CP9zxbLM+7mMxiWQwn9IsSQvwAGG6y6z1Sym/o56+i3ho4CmyWUk4IIS4GbhRCnLvA930D8AaAzZs3L/zGLUvKO796n9ficbZYIjZPaqPBcw2Vys0tgqwSgkQ4yP7xHFUJCR3UNXn/jRaBCTBv6W/uGvJbBB+79iJK1SpD6Zh3TEjX2zGuIWMRnLEuTdYpe9cfTEc8UVF9gmtCcO6GLgZSEb748wNcfvYQM4X5YwSNxMIB/b2oe73htn0UShX6khGOzxatRWBZFk7oFyWlvHy+/UKIEPAbwMW+cxzA0c/vFkI8AZwBHAY2+k7fqLc1e9/rgesBdu3atXTtqywd88RYlg3dcYSAnzw2xpb+BAcm8mSKZYbS7c+f9VkEzdYRTOQczhxOEwsFvfo+xiKoVQ+ttwimci7BgGB9txrcTXqounbZ8/fHwoGWg3NXPOz1JDg6UyQSCrCpL8Gd+ye9nsK9iQg9iQgXbe7h4s19pGMhLtzUw9O39RELB3n9c7bzoe8+wi2PjyNl6zUEzWi0CA5PFRjpibOxN66EoFied1WxxbIYlts1dDnwiJTSc/kIIQaFEEH9fDsqKLxXSnkUmBVCPEPHFV4DfKPZRS0ri1Ou8Ksf+xmfvX0/UzrDZoce/Y3Lp5FiqVLXitJYBKY6J8xdUNafjBKPBL3txiUU91xDcy2C3kSEtA64jvoa0efdireOIBpqbbH0JGr9AY5MF1jfrXoB55wykzm1vS8ZoS8Z4etveTab+xP0JiPc+NZns20gCcBvPWMzXbEQr/vsnQB1Vkc7jBAY62imUKI7HvZ6EWd9JTEslqViuW3Ma5kbJH4e8H4hRAmoAm+SUk7qfW8BbgDiwHf0w7LKOD7jUChVODSVZ0oPjsYdkyk2L+P8kn/8Ka+6ZDNvev5pQC1Y7F9BbGbB5UqV6UKJvmSkbpWtKYZmLIO5Reoc+pJhz3UynnWIh4OqeqlTwSQjG/dLM/pTUS+t1AhBMhoi65Q9i8Nk9rQiHQvz/qt28tPHx3jRWeu44px18x7vJ2qEoFQvBENdUV1Ib/7yEhbLYljWX5SU8rVNtn0NlU7a7Pi7gJ3LeU+WE8f4zidzrmcR1IRgrkVQrUoOTOR5/HjWe22Kp5mc/Ugo4A3sU/kSUqqaQP61AMYiMLGCZtVK+5IRb4FYVcJAOsLByQJ5t4JJj5/PItjYG+dBXTfo4FSBy84YJB0LUapIjs0W6U2E513Va7j6ohGuvmjhuQ6NrqGZQonh7hhD6RhSG1TWNWRZauzKEcuCOaoXWU1kXW9Gb+rbNLMITKkH42PPumVvUDMZOn0JNfuvVmvlmPuSkbrSDMYiiARV0Tf/ymF1feVO8hdNS0fDREMB8q4KSgsB4WDrgXxjb5zJnPpcYxmHTX0JLybx5GS+49aPiyXekD46Wyxr11CtsY21CCxLjRUCC8VShSv+/if86JHRjo4/0swi6KtZBHftn+TtX7nXW/lrZv9mgPcXdZvSQmAWcxXLFSa0YPQno3VCYJ4LIUj4YgcGYxEEAsIbvFPREMloSGcNVYmFgi3LPUCt5+/P907o13FS2hV0cDK/4EJuC6Uxa2imUKJLxwgMaZs1ZFlirBBYlNtmNMtdBybbH0xtsdRkzvViBBt7EwihZrA/fGSUr99z2KvCaer0THhCUJvJTzWUac67Fc8d1J9S6wgMyYbn/mBxpSqZ9qVqGushEQ16olEsVYjOEx8A2NSr1gHcroVgU1/CC84emy3OuyZgKfAHi4ulCm65qmIEaWsRWJYPKwQWDuoVtsdmnDZHKrwYQd5lPOuQjoWIhAKkIiEyxZI3kJtG85lGi6DotwjqyzQX3Eqdayjptwii9c9zPotgOq9W/ZqBOuULLCciQfJOBaekLIL5MBbB7U/ULAIjKlIuvLTzQjFrMIrlimc5dcXCdZlHdh2BZamxQmDh4JQSAn+v3fkwMQIp1QpaM4inYyEyxdrCqydGa01ioLaK2O8amm5wDeXdChO6zlBvIuL5zKHeIkhEgvWNbBrq/nsWQSRIIqJcQ8Vye4tgIBUhGgrw+GiWSDDAunSsbuBdaPvHhRINBRBCVWI1C9u642HikSBdMdNc3gqBZWmxQmDxSjMcW4AQmEHpibGs59ZJx8LKItA+fn9JaMNEzvUyi/qStdaQNSEoM5lz6E1ECAaEVxkUam4TUBVDcw3XNdeEWmnpZDREMqpEoxOLQAjBRu0eGumNEwiIeiFYZotACEEspFJe/UIAeMXtbIkJy1JjhcCyIIugWFKum50jqj7P0ZmiNzimtEUw6VkEuklMsb7hjHENreuKYdaYGTEplCpeeQnwpYxGgnVpm8mGYHGjRWBSLBORIPFwiJxb6cgigFoGlBEEv09+uYUAVOZQsVT1hKBLC4HJHLLN5S1LjRUCixcjyBTLc1brNmLcQkYIoDabN66hyRYxAlClI0yw2J8S6QmBdg2ZATfuCUH94JdoCBYbIajFCBotgjJOqUo01P4nbwTAxAv8A+9yxwgAYqEAhVLFE0zPIkjHiIUDhG2/AMsSY39RaxwpJYemCt5gYxqxtMIEis/d0OVt6/O5hiayDhmnTG8i7OXj+104kzllESQjwTqXS2+yFiNQ6wHqLYLG9omN6aNGCOZkDXkxAlWGupOCeEYAjCD4YxPLHSMAiEW0ayhfLwRX7hzmlbs2Lfv7W9YeVgjWODOFElmnzK4tvUD7OIFJHfULQa8vWHxUn//0rX2AihNkdcN10EKgc+P9awR6EvVZQ6bxS8IrPV1vESSjoTkCk46FvNny3KyhMoenCx0tCDMCYFxEKlah7rUvdRJcQ+EgTqnCjLacTDzmJecO8/6r7MJ7y9JjhWCNYwLFF29VQjBfnEBKyR3eQquE5xLyZw2ZFcOXbNNCMJolUyzTn4wQDgomtEXQFQvXDe5mpp1xyrqrl3IbeRZBQ/P3eBOLwJ/j719HkIyoVNPjsw7P2N7f9jvZtaVPVRPV34n/eifFIgjXgsXJSNC2jrQsOzbqtEapViVf/MWTXiqnmcGbvgJSSrUSV7tSpJS8/Sv3cuO9R3jVJZuJhYM666dErxYEfzG28zf2EAkF2DeeI+uUScfCBIRgMquyhrriobrUUDNTPzpdqFsPEA4GCAeF14PAkIwEKVclebfM3QemODxdqAvk1lkEvnOfc/pA2+9muDvGjW99dt22dFStkYhH2ruWTpR4OEjeVe0ubSN5y8nACsEa5at3HeTPb3zQe33mcJpUNOTFCD74rYf5zoPHuPVdLwTg3oPT3HjvEd582Wn8yUvOBNRgvXcs50sfrf2cBtNR1nVFdQ39EqloiGgo4FkE69IxEj5/fToWQohaBpN/UI+Hg3MsAmNNfO2Xh73PcfnZQ97+VF2MQJ27sTfOZl0cb6EkoyGvYf1yEwsHmci5XnkJi2W5sUKwBpnMufztdx/hgk09OKUKebdCVyzsDdz7xnPccNt+ylVJplgiHQvztV8eIhoK8ObLTvNq9ZjB2u8aMvQlI6xLqxr65WqVVFQN9JM6a+j0wZpFEAkFCAQEiXCQQ1PKVdXv88X3JSNeDMFgBveHjswC8AcvPL1utt+trZTuRM0F1Yk10IqTmbsfCwd0jMAKgeXkYIVglSGlnLco2lLwmVv3kSmW+dDLz+P0wZRXqmG4O8aR6QJ/+52HKesE/2O6S9c37zvKlTuH69w/poG7FyzWufvhoKArFmJdV4yHj80SCQbY2JsgFg5w1/4pz+VhhMCkdI70xtmtB3b/7Pv61+zy4hEG4+55YjRLfzLCO198Zt3+F541xCd/82mcuS7NvjG1nuFZJyAE73zxGbjlk9MMLx4OknXKXnc0i2W5sVGoVcShqTxn/vl3vXr48/Fvt+zlXV+7f1Hvc3i6wHBXjLOGuwgFA54fel1XjPsOzXDz7uM8/4xBQK0b+J+HR5kplHj50zbWXWekJ04sHKAnXltHAGoGL4RgMB1lbNbRMYIQ/ckIx2aLFEsVrq74IaEAACAASURBVL5oxJvVmzjEv73m6bzgzEG6YiE29NRq65yxLj2ny5dxFT0+mvEaxvsJBwO89Lz1CCG4dHs/v/WMzbzorKE5x3XKxVv6eOZp7QPNS8E5G7oYzTjsHcvZGIHlpGCFYBXx+GgWt1xl/0Su7bG37hnn5t3HFvU+eacyJy8f4PKz1/GM7X184jefxl/pNMVjM0Vue2KcdCzEsxtm1K991lZuettzvKwW0yLSZPys64qRccqMZx1S0ZC3/fefu52LNvcSDyvhMBbB5v4En/ndS7jvvS/2rtUKY01M5Ut1otGMvmSED1x93imzIvfqC0eIBAO4lWrbbmgWy1JwavzPWCOMZ1TGTmOd/WZMF0pM5UsUS50tkvKTc8tNB8VfOW89v3LeegBc3VD+6IyKGWwfTBFs6MyVjIY4Y12tU72xCAa0f9+sHC6WqqRiIV509hCHpvK8/YozgJqfv3G1byeuMf8ir5Gep5b7pDcZ4cqdw9x03xFrEVhOCtYiWEWM69IMhQ6EwKw67bRiqJ+sU25bwTISCjCQinJstsD+8TzbdWP2+eiK1a8r8DdTSUVD7Bzp5iOvuMATrkbX0ELwWzTtLIJTkWsvUSuIzYpri2U5sUKwihjXTdNzber9gLIIoH1JiGbknHLdqt5WDHdH2Tee4/B0ga397YUg5YsRAHXNVJpl3TQGixdCvM4imBsjONV55vZ+/u4VF/Br529Y6VuxrAGsa2gVMaZdQ+0sAimlV5my09LRfnJOpSN/+XBXnFseHwNg22B7IQgGBO9+6Vk8Z4eKJQz5LIJkZO77mbTO+ZrJt8K/rqBZsPhURwjByy/e2P5Ai2UJsEKwijAWQbsYQdYpU/Gldy6UnNveNQSwvjuGo2MF2zqwCADe+PzTvOddsRCxcMCLETTixQg6KA0999za9Z6KQmCxnEysa2gVUROC+V1DpqsXLNYiKM8p4taM4e7ajH7rwMIDskIIL07QrOG6cQ21axbTjEgoQCggiIQCy95H2GJ5qmOFYBVhgsXtLIIZX6vHhQaL3XKVUkV69frnY70WgoFUtG06ZytMnKCpRRBevEUAyqLY0B2ra1hjsVgWjhWCVUK5UvUaubcTAmMRREMBr1FMp5jSzR3FCLQQdJIx1AoTJ2jmigoFA0SCgUUFi0F9hpFe6xayWE4UGyNYJUzmXK+Ec1vXUEEJxhnr0hxfoBBkFyAE67vVILvtBIRgnV4R3MwiALXmYLFdv567Y4AdQ+n2B1oslnk5YYtACPEKIcRuIURVCLGrYd+7hRB7hBCPCiFe4tt+pd62RwjxLt/2bUKIn+vtXxFCrBnn75iOD0DnFsGZw2mOZxwvcNwJ5trNsngaWd8dIx0NsXNjd9tjW7FjXYpUNNRyheyX3vAM3vqC0xd17Q9fcwG//7zti743i8WiWArX0IPAbwA/9W8UQpwDXAucC1wJfEIIERRCBIHrgJcC5wCv0scCfAj4Bynl6cAU8PoluL9TAhMfGEhF2qaPmhjBWcNpKlXJhE9E2lGzCNrHCGLhID/5kxfw6ks2d3z9Rl65axM/+ePLWi4a29KftGUULJYV5oSFQEr5sJTy0Sa7rgK+LKV0pJT7gD3AJfqxR0q5V0rpAl8GrhKqrsALgf+nz/8scPWJ3t+pgikvsbkv0YFF4BILB9iiUzoXkjlkYgSdpI+CWhzWWFpiIQQDwqtSarFYVifLGSweAQ76Xh/S21pt7wempZTlhu1rAuMaUkIwf4xgplCiJx5hWAdiFxIwNtfuJH3UYrGsDToaDYQQPwCGm+x6j5TyG0t7Sx3dzxuANwBs3rx4t8VqYjzjEA8HGUhFW1oE37r/KIen80znS/Qkwgzpom7jLVxDP3p0lPsPzvBHl+/wtmUdde1OLQKLxfLUp6PRQEp5+SKufRjY5Hu9UW+jxfYJoEcIEdJWgf/4xvu5HrgeYNeuXSenW8gyM551GEhHSERU4/JqVc7Jj//6PYe47YkJzhpO0x0Pey0ixzNu02veeM9hvnHvEV509hA7R1TAN7eAGIHFYlkbLKdr6CbgWiFEVAixDdgB/AK4E9ihM4QiqIDyTVJKCfwIuEaf/zvASbc2VoqJnEt/Mko8EkJKKJbnWgV5V7WVfODwDD2JMJGQaiozkWtuEWSKatC//qd7vW0LSR+1WCxrgxMeDYQQLwP+GRgEviWEuFdK+RIp5W4hxFeBh4Ay8FYpZUWf8zbgZiAIfFpKuVtf7v8CXxZCfAC4B/j3E72/U4XpfImBVMSbqefdColIiPd/8yEOT+f5l9/e5bWULFWkV6d+IBVhItvcIpjV2UXfeuAoI71xdm7oJu+WCQbEohdxWSyWpx4nLARSyq8DX2+x74PAB5ts/zbw7Sbb96KyilYl1arkl09OsWtr35Jfe7rgcvpQirhOszQppHc/OcVM3vQpqAWRTTP3/lS0ZYwgUyxzwaYe9o1l+eSPnyAVDXHNxRtJRoLL3hfZYrGcOthp4QL4/sPHueZTt/PoscySX9sEgE02j+lJcHiq4AV4c07NXVRnEeRciqUKb/3iL9k7lvWOmS2WOGMoxT1/8WLe+2vnkHXKPDmZt24hi8VShxWCBfCEHmSfnMwv6XXLlSqZYpmeeISEzzVULFUYzzpeymehVPEshp6EEoL+pLIIHjo6y7ceOMqte8a962aKZdKxMMGAYEu/qh760JFZKwQWi6UOKwQL4NBUAVhc6ef5MCuFexJhryJnwa146wPyrsoiyrtlnnfGAOGg8DqG9aciTOdL7BtTDe8ncsqNVKlKsk6Zrrga9P2Lz6wQWCwWP3ZEWAAHtSWw0EJv7Zj2CYEZpPNuhcNaeACybpliqcrZ67v48DUX0OU1ildrCe47NA2o4nUAWZ0xZMpHb+yNIwRISUclqC0Wy9rBWgQLoJ1FcMOt+/jqnQeb7ts3nuPC93+vzodvMEXkuuNhr1lL3i1zZLomBKYERSISpDse9oK9AykVNL7nSSUExiKYLaprml7B0VCQDbqaqF1VbLFY/Fgh6JBqVXoz9GbtIatVyT/+z+P80/88jpSSw9MFjs7UBvLdR2aYzpd44PDMnHNndFnpnkTEa9+Ydysc8gnBmCcE9YO4qePz8NFZACaz9ULgL+i2uU/FCeyqYovF4scKQYeMZhzciurf28wieHw0y3S+xOHpAk9O5vmdT/+CP/2vB7z9RjwO+dw9BmMR9PqyhhpdQ6YWUSJS79YxbRrLuhS1cQ2ZxWRdvj4Apt2kXVVssVj82KlhhxycUvGBbQPJpjGCn++b8J7/+8/2sWc0S8zXgnFUz+j97h7DlBaCnrjPInCUa8j49ccyLYSgobLnpF5zYBaTdcX9FoEKGHfSi8BisawdrEXQIYe0EOza0kvGKXs1eww/3zfJ+u4YQ+koX7jjAACzhdoxxiJoJgQzeRchlD8/HAwQDgrypQqHpwueO6eVa6grFiISVH/G4a4YUzkXKaVnEaR9FoFJIbVZQxaLxY8Vgg45OKkG8Iu39AL17iEpJb/YN8ml2/p41mn9mIZhxk8PtSbzh5sIwXShRHc87BWZS0RC5JwyR2cKXivG8RauISEE/TpgfOGmHspVyWyh7AsWz40RWCGwWCx+rBB0yKGpPEPpKJv1rNofMN4/kWcs43DJtn6eddoAACM9cWYLJaRuROwJwVTB22aYzpfo8blwEpEgBybylCqSM9algNYWAVATgs09AEzknKYWwelDKZ53xiC7tJhZLBYL2BhBxxycLLCxN+41g/ELwe4jKhPowk09bBtIMpV3KVWqfPR7j5FzKyQjQY7NFokEA+TcCrPFslciAmAq79KdqLVnjkeC3H1gCoDzdPnoVsFiUKuLo6EAZw0r62Ey55IploiHg4SDNa2PhYN87nWrtpSTxWJZIaxF0CHHZous74kz3B3zXhuM339jX5x4JMgbn38ag2kVxJ0tlJgtqsVg5450AdRlA4FaWdybqAlDMhIi65TZNpDkhWcPAT6LoEnGz4WbenjG9n5vcdlEzmW2UFtVbLFYLPNhR4oOmcg6DCQjJCIhumIhz9UDamBPx0J1Ofvm+Uyh5PUAeNrmXu55cpoj0wXO2dDlHTudL7F9IOm9NovKXvfsrURDQSKhgNfcvplr6O1XnAHUBGky55JxSnXxAYvFYmmFtQg6oFSpMlss06tz9jf0xNl9ZNbz9R+eLjLSE687x6RtzhZKnhvpaZuVb/7ITL1FMJ13vbLSoDKBuuNhXn7xRuD/b+/+YySv6zuOP9/za3dv927vN3cc3B3IIVFJEZYfJkKMoCJRT1AEQwWjlZjWtGrahoZoTGvSUtM2sWlqz6IFg0oqvUJCDIoWTdpIC3jCIZ4ciHI/kb3b3eN2dmdm590/5vPd++7czuzszX3nu+339Ug2+73Pd2bnne/Mfd7z+fmFwVKemTACHW06N5/VIb4joUUQHx8QEWlFiaADR8Pc/Gjx1k2Xns2Tvz7KY3t+CzS+iZ/ZlAiiMYCJqdps6+ENZ66gVMjN6RqaqftJYwZ/8q4L+NpHL5399h/N8ukv5sjnWt9HoL+YZ1kpz+hrjTGCFWoRiEgHlAg6cPR4WPkbEsEtl29hy5pl/OV3n2Om7hwYL5/cIug/0SKIEsHG4X7OHO6fM4U0vvNo5PUbls9OU4UTC8A62SNo9WCJI8enmZhSi0BEOqNE0IHonsBR10upkOMz15zPLw+/xmN7XmFssnpSiyAaqJ2YqnJ4YprhgSL9xTwbhwdmt5eGRrcQzE0EzaIB4vlmDDVbM9i4Uc2xqeqcVcUiIq0oEXQgahFEiQDgba9fB8CDuw4AcObK/jnPiTZ2myjXODQxNTvtdO3yvtn9gOBEi2C4TaUd/a1OEkGjRaAxAhHpnBJBB440tQigsVPotvVDfP/nh4HGfv9xhXyOob4CE1NVDoyVZ6edrhkszbnH8HStsZFdf6F1JR8lgE66hrasGWTPoWNUZuoaIxCRjigRdOBINEYQm9kDMLJ1FeVq4z7CzV1D0Jj9M16u8pvRydntHdYMljg2VWO61nheJSSCUqH1WzG4iBbBZ645n9etG5p9fRGRhSgRdODoZIUVYUO4uEu2rAagkDPWL+8/6XkrBor85sgkx6Zrsxu+RbuFRt1NHSWCUueJYHhZkXs+dhlXX7CeS89ZveDjRUT0lbEDo8crc7qFIpdubczs2TDcP++0zhX9RZ4NN6KJWgTR33n1tWk2DPdTDfc4aE4ycSdaBJ29XRuG+7n7o5d29FgREbUIOnC0RSLYvHoZa4f65u0WgsbMoeOVRhdQdPP46NaS0YBxdLOb9i2CzmcNiYgslloEHRg9XmHTypO7fsyMP9/+xpazc+LTN89e3UgWUUKJpqRGg8WlNi2CZYtsEYiILEZXLQIzu9HMnjWzupmNxMrfYWZPmtkz4ffbY+ceM7M9ZrYr/KwP5X1mdr+Z7TWzx81sazexnU6tWgQA1124kSu3rZv3XDRrZ93yvtlKPBojGA17B1U7aBEMLWIdgYjIYnX7FXM3cAPwT03lrwLvdfcDZvYm4BFgU+z8Le7+RNNzPg4cdffzzOxm4C7gpi7j65q7c+R4ZXZV8WJELYItYXwACIPOxmjUNdRJiyAaLNa9hkUkAV21CNz9OXffM0/5T939QPjns8CAmfU1P67JduCecPwd4Goza72xTo8cr8xQmanP7jO0GNH0zc2xRGBmrB4sMRrWEnQya2h2QVmbDedERE5VLwaLPwA85e7TsbKvh26hz8Uq+03AywDuXgPGgTU9iK+tI6ELp3kNQSeiFkF0V7PImsETq4s7SQSLWVAmIrJYC9YsZvYosGGeU3e6+4MLPPeNNLp43hkrvsXd95vZcuAB4CPAvZ2HDGZ2O3A7wObNmxfz1EU7Eu08OnQqLYLQNdScCIZKs/cXiMYICm12FY0SypAWiIlIAhasWdz9mlP5w2Z2FrATuNXdX4j9vf3h9zEz+yZwGY1EsB84G9hnZgVgGBhtEdMOYAfAyMiIz/eY0yXaXuJUWgTnrR+kVMhx4aaVc8rXDJb49egkANMzdUqFHO16wS7YsJy7PnAhb79g/aJjEBFZSCJdQ2a2EngYuMPd/zNWXjCzteG4CLyHxoAzwEPAbeH4g8APvfku7ymIbhEZ3QZyMc5bv5w9f3Et560fmlO+erBvzhhBX5uBYmiMK9x06Wb6NUYgIgnodvro9Wa2D3gL8LCZPRJOfQo4D/h80zTRPuARM3sa2EWjFfDV8Jy7gTVmthf4LHBHN7GdLgfGpjCDM1acvI6gE/N9018zVOJ4ZYap6gzVmTrFNuMDIiJJ66rT2d130uj+aS7/IvDFFk+7pMXfmgJu7CaeJBwan2LtUF/bwdzFilYXjx6vUKnV204dFRFJmmqgBRwYL3Pm8Km1BlpZPRgtKptuJAK1CEQkRaqBFnBofGr2XgKny6pwN7KxySrVGVciEJFUqQZawKHxKTYOz7+p3KmKBn3L1Rmma/W2O4+KiCRNNVAbx6aqHJuusfE0twgGwgKxqWpj1bJaBCKSJtVAbRwKN5k/3V1DUYtgqjpDtYPpoyIiSVIN1MaBkAha3W/gVPWHFsBUtU5lpk6xkPqWSiKSYUoEbRwaLwOw4RTXELQSdQ2VqzOaPioiqVMN1Ea3i8la6S/Exgg0fVREUqYaqI0kFpMB5HJGqZCjHFYWlwraOkJE0qNE0MbBianTPmMo0l/IMV2th+mjGiMQkfQoEbRxaLx82scHIgOlPOVw05s+dQ2JSIpUA7UxNlk9pe2nO9FfzDNVC11DGiwWkRSpBmpjvFxlOGwHcboNFEOLQCuLRSRlqoFamArbPwwPJJMI+op5pmp1zRoSkdSpBmphYqoKnLgB/ek2UMxRrtSo1bXpnIikK9M1UL3u7Ds6Oe+5iXINOHG/4NOtv5iffQ0lAhFJU6ZroO/9/DBv+9Jjs7eNjBsvhxZBUomgkJ9tdWiwWETSlOkaaN/RSWp1ZyxU+nFRJZ3UGMFAKc9EeF21CEQkTZmugcYmGxVxpVY/6VxUSa/oT6prKMfxygygFoGIpCvTNVDU/TPdJhEk1SKItqIGNH1URFKV6Roo6hKar0VwYowgmVlD8USgriERSVOma6DxBRJBfzFHX0Ibwg0oEYjIEpHpGmh8sgJAZWbmpHMT5Vpi3ULQGCOIKBGISJoyXQMt1DWU1EAxNLUINEYgIinKdA3UdrB4qppoi6BPXUMiskR0VQOZ2Y1m9qyZ1c1sJFa+1czKZrYr/Hwldu4SM3vGzPaa2ZfNzEL5ajP7vpk9H36v6ia2hdTr3jYRjJeriS0mg6bBYrUIRCRF3dZAu4EbgB/Pc+4Fd78o/HwyVv6PwCeAbeHn2lB+B/ADd98G/CD8OzHHpmu4N47nXUeQcItgQNNHRWSJ6KoGcvfn3H1Pp483s43ACnf/ibs7cC/w/nB6O3BPOL4nVp6I8ckTq4nnHSOYrCa24RxosFhElo4ka6BzzOynZvYjM7sylG0C9sUesy+UAZzh7gfD8SHgjARjY6xcmT2uzMxNBPW6c2w62VlD8RaB7lAmImla8CuvmT0KbJjn1J3u/mCLpx0ENrv7qJldAvy7mb2x06Dc3c3M28R0O3A7wObNmzv9s3OMl1u3CKJuo56NESgRiEiKFkwE7n7NYv+ou08D0+H4STN7ATgf2A+cFXvoWaEM4LCZbXT3g6EL6ZU2f38HsANgZGSkZcJoZ6xN19BEwjuPgraYEJGlI5EayMzWmVk+HJ9LY1D4xdD1M2FmV4TZQrcCUaviIeC2cHxbrDwR8RbBdK2xoKwauojGE95nCDRGICJLR7fTR683s33AW4CHzeyRcOoq4Gkz2wV8B/ikux8J534f+GdgL/AC8N1Q/lfAO8zseeCa8O/ERJV9fzFHpVbnv/a+yoVfeITR16ZjdyfTgjIR+f+vq2kx7r4T2DlP+QPAAy2e8wTwpnnKR4Gru4lnMcYmK/QXcyzvL1KZqfPS6CRT1Tr7x8qJ7zwKzV1DltjriIgsJLn5kUvceLnKyoEShbwxXatTrja6h8Ymq7PjB8PLkk8EpXyOsKZORCQVmU0EY5ONBWPVep1Krc5UlAjKVY6GRLAqwUSQzxmlfE7jAyKSuszWQuPlKsPLipTyjTGCcrhb2Hi5ythkhVIhN6cfPwl9RSUCEUlfZmuhRtdQkb5CjulanckoEUxWODpZYdWyYuJdNgPFvMYHRCR1mU0EUddQqRBaBLExgqOTVVYtKyUeQ38xrxaBiKQus7XQfZ+4nD+8elsjEczMHSMYm6ywMsHxgchAMa+poyKSuswOFr9u3RDQmLUzUa7NjhFELYJt64cSj6G/mGO6lvjLiIi0ldlEEOkr5Od0DU3Mtgh60zUkIpK2zCeCqGsoSgRHJyuMTVYTnToaee/vnDnbJSUikhYlgsLc6aP7x8rU6t6TweLfvWJL4q8hIrKQzI9Ulgo5pmszsy2CaBppLwaLRUSWAiWCfGMdQdQiiPSiRSAishRkPhH0ha6hqeoMhdyJxV2rBtUiEJFsyHwiiAaLJysznLGif7a8F7OGRESWAiWCfA53KFdn2Dh8IhGoa0hEsiLziaAvdqewDSERmCV7LwIRkaUk84kgvsVD1CJY0V8kn9NmcCKSDUoEhROrezcMDwDJ3odARGSpUSKI7f65dqhEIWcaKBaRTFEiiCWCgWKelcuKahGISKZoi4nYGMFAKc/Fm1fxpk3DKUYkItJbmU8EfbEWwbJSnh23jqQYjYhI76lrKJYItC20iGRR5hNBX9MYgYhI1mQ+EcwZLC4pEYhI9igRqEUgIhnXVSIwsxvN7Fkzq5vZSKz8FjPbFfupm9lF4dxjZrYndm59KO8zs/vNbK+ZPW5mW7uJrVPNs4ZERLKm2xbBbuAG4MfxQne/z90vcveLgI8Av3L3XbGH3BKdd/dXQtnHgaPufh7wd8BdXcbWkahFkLO5SUFEJCu6qvnc/Tl337PAwz4MfLuDP7cduCccfwe42swS3/AnSgQDxTw9eDkRkSWnF1+BbwK+1VT29dAt9LlYZb8JeBnA3WvAOLAm6eD68o3uIHULiUhWLbigzMweBTbMc+pOd39wgedeDky6++5Y8S3uvt/MlgMP0Og6uncRMWNmtwO3A2zevHkxTz1JtA211hCISFYtmAjc/Zou/v7NNLUG3H1/+H3MzL4JXEYjEewHzgb2mVkBGAZGW8S0A9gBMDIy4l3ENzsusEwtAhHJqMS6hswsB3yI2PiAmRXMbG04LgLvoTHgDPAQcFs4/iDwQ3fvqpLvRC5nFHKmqaMiklld7TVkZtcDfw+sAx42s13u/q5w+irgZXd/MfaUPuCRkATywKPAV8O5u4FvmNle4AiN1kRPlAo5dQ2JSGZ1lQjcfSews8W5x4ArmsqOA5e0ePwUcGM38ZyqUiGnwWIRySxNnKcxTqCuIRHJKiUCYOPKAc5cOZB2GCIiqcj8/QgA7vu9yynoZvUiklFKBMBQny6DiGSXuoZERDJOiUBEJOOUCEREMk6JQEQk45QIREQyTolARCTjlAhERDJOiUBEJOOUCEREMk6JQEQk46wH935JlJn9Fvj1KT59LfDqaQzndFmKcS3FmEBxLcZSjAkU12IsNqYt7r5uoQf9n08E3TCzJ9x9JO04mi3FuJZiTKC4FmMpxgSKazGSikldQyIiGadEICKScVlPBDvSDqCFpRjXUowJFNdiLMWYQHEtRiIxZXqMQERE1CIQEcm8TCYCM7vWzPaY2V4zuyPFOM42s/8ws5+b2bNm9keh/Atmtt/MdoWf61KI7SUzeya8/hOhbLWZfd/Mng+/V/UwntfHrscuM5sws0+nca3M7Gtm9oqZ7Y6VzXttrOHL4bP2tJld3OO4vmRmvwivvdPMVobyrWZWjl23r/Q4rpbvm5n9Wbhee8zsXT2M6f5YPC+Z2a5Q3str1apOSPbz5e6Z+gHywAvAuUAJ+BnwhpRi2QhcHI6XA78E3gB8AfjjlK/TS8DaprK/Bu4Ix3cAd6X4Hh4CtqRxrYCrgIuB3QtdG+A64LuAAVcAj/c4rncChXB8VyyurfHHpXC95n3fwuf/Z0AfcE74v5rvRUxN5/8G+HwK16pVnZDo5yuLLYLLgL3u/qK7V4BvA9vTCMTdD7r7U+H4GPAcsCmNWDq0HbgnHN8DvD+lOK4GXnD3U11I2BV3/zFwpKm41bXZDtzrDT8BVprZxl7F5e7fc/da+OdPgLOSeO3FxtXGduDb7j7t7r8C9tL4P9uzmMzMgA8B3zrdr7uQNnVCop+vLCaCTcDLsX/vYwlUvma2FXgz8Hgo+lRo6n2tl10wMQ58z8yeNLPbQ9kZ7n4wHB8CzkghLoCbmfufNO1rBa2vzVL6vH2MxrfHyDlm9lMz+5GZXZlCPPO9b0vhel0JHHb352NlPb9WTXVCop+vLCaCJcfMhoAHgE+7+wTwj8DrgIuAgzSaqb32Vne/GHg38AdmdlX8pDfapT2fcmZmJeB9wL+GoqVwreZI69q0Y2Z3AjXgvlB0ENjs7m8GPgt808xW9DCkJfe+xXyYuV80en6t5qkTZiXx+cpiItgPnB3791mhLBVmVqTxht/n7v8G4O6H3X3G3evAV0mgabwQd98ffr8C7AwxHI6aneH3K72Oi0ZiesrdD4f4Ur9WQatrk/rnzcw+CrwHuCVUIoSul9Fw/CSNvvjzexVTm/ct1etlZgXgBuD+WKw9vVbz1Qkk/PnKYiL4H2CbmZ0Tvl3eDDyURiChL/Ju4Dl3/9tYebyP73pgd/NzE45r0MyWR8c0Bhx307hOt4WH3QY82Mu4gjnf1tK+VjGtrs1DwK1hdscVwHisiZ84M7sW+FPgfe4+GStfZ2b5cHwusA14sYdxtXrfHgJuNrM+MzsnxPXfvYoLuAb4hbvviwp6ea1a1Qkk/fnqxUj4Uvuhzt5n7wAAAMdJREFUMdL+SxqZ/c4U43grjSbe08Cu8HMd8A3gmVD+ELCxx3GdS2Pmxs+AZ6NrBKwBfgA8DzwKrO5xXIPAKDAcK+v5taKRiA4CVRp9sh9vdW1ozOb4h/BZewYY6XFce2n0IUefr6+Ex34gvLe7gKeA9/Y4rpbvG3BnuF57gHf3KqZQ/i/AJ5se28tr1apOSPTzpZXFIiIZl8WuIRERiVEiEBHJOCUCEZGMUyIQEck4JQIRkYxTIhARyTglAhGRjFMiEBHJuP8FU9DCl+PLQpoAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 1440x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "while frame_idx < max_frames:\n",
    "    state = env.reset()\n",
    "    episode_reward = 0\n",
    "    \n",
    "    for step in range(max_steps):\n",
    "        action = policy_net.get_action(state)\n",
    "        next_state, reward, done, _ = env.step(action)\n",
    "        \n",
    "        replay_buffer.push(state, action, reward, next_state, done)\n",
    "        if len(replay_buffer) > batch_size:\n",
    "            soft_q_update(batch_size)\n",
    "        \n",
    "        state = next_state\n",
    "        episode_reward += reward\n",
    "        frame_idx += 1\n",
    "        \n",
    "        if frame_idx % 1000 == 0:\n",
    "            plot(frame_idx, rewards)\n",
    "        \n",
    "        if done:\n",
    "            break\n",
    "        \n",
    "    rewards.append(episode_reward)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:pytorch4]",
   "language": "python",
   "name": "conda-env-pytorch4-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
